{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss 0.0049, train acc 0.832, test acc 0.897\n",
      "epoch 2, loss 0.0036, train acc 0.898, test acc 0.904\n",
      "epoch 3, loss 0.0036, train acc 0.903, test acc 0.910\n",
      "epoch 4, loss 0.0035, train acc 0.906, test acc 0.911\n",
      "epoch 5, loss 0.0035, train acc 0.908, test acc 0.910\n",
      "epoch 6, loss 0.0035, train acc 0.908, test acc 0.913\n",
      "epoch 7, loss 0.0035, train acc 0.908, test acc 0.913\n",
      "epoch 8, loss 0.0035, train acc 0.909, test acc 0.915\n",
      "epoch 9, loss 0.0035, train acc 0.909, test acc 0.915\n",
      "epoch 10, loss 0.0035, train acc 0.908, test acc 0.916\n",
      "epoch 11, loss 0.0035, train acc 0.908, test acc 0.912\n",
      "epoch 12, loss 0.0035, train acc 0.909, test acc 0.910\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEGCAYAAABCa2PoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3Rddb33+/c3a+XSJG1K07RA05JCE6DcipQCG9kIbKRcDhXlUhUFhc3weeARtw9qe/bWvWUczsGjB3BvER/A8iCwrVhEuwUtClRkCL1hVQqUhlJoyqX3S9rmspLv+WPOJHOtrlyarpWVtfJ5jZGRueb8zZnfbCGf/i5z/szdEREROVRFua6AiIgUBgWKiIhkhAJFREQyQoEiIiIZoUAREZGMiOe6Ark0fvx4r6ury3U1RETyyqpVq7a6e03q/hEdKHV1daxcuTLX1RARyStm9k66/eryEhGRjFCgiIhIRihQREQkI0b0GIqIFJ729naamppoaWnJdVXyXllZGbW1tRQXFw+ovAJFRApKU1MTo0ePpq6uDjPLdXXylruzbds2mpqamDp16oDOUZeXiBSUlpYWqqurFSaHyMyorq4+qJaeWigHqaW9g8bNzazbvIdTasdydE1lrqskIikUJplxsH+OCpSD9G+L17BwxUYAvnnZdAWKiEhIXV4HqX7i6O7tdR/uyWFNRESGFwXKQWqY2NMieVOBIiIpdu7cyQ9/+MODPu+SSy5h586dB33e9ddfz6JFiw76vGxQoBykhqQWSjNa8VJEonoLlI6Ojj7Pe/rppxk7dmy2qjUkNIZykCaMLmVMWZzdLQn2tCb4YHcLR1SNynW1RCSNunlPZe3aG+68NO3+efPm8dZbbzFjxgyKi4uprKzkiCOOYPXq1bz22mt84hOfYOPGjbS0tHDrrbdy0003BXUN3y3Y3NzMxRdfzEc/+lH+9Kc/MWnSJH71q18xalT/v2eeffZZbrvtNhKJBKeffjr33XcfpaWlzJs3j8WLFxOPx/n4xz/O9773PX7+85/z7W9/m1gsRlVVFS+88MIh/5koUA6SmVE/cTSr3tkBwJsfNitQRKTbnXfeyauvvsrq1atZunQpl156Ka+++mr3sxwLFixg3Lhx7N+/n9NPP51PfepTVFdXJ11j3bp1/PSnP+WBBx7g6quv5oknnuDaa6/t8+e2tLRw/fXX8+yzz9LQ0MDnP/957rvvPj7/+c/z5JNP8sYbb2Bm3d1qt99+O0uWLGHSpEmD6mpLR11egxAdR9HAvIj0ZdasWUkPBv77v/87p5xyCmeeeSYbN25k3bp1B5wzdepUZsyYAcBpp53Ghg0b+v05a9euZerUqTQ0NABw3XXX8cILLzBmzBjKysq48cYb+cUvfkF5eTkAZ599Ntdffz0PPPBAv91xA6UWyiDUT0geRxGR4am3bqmhVFFR0b29dOlSfv/73/PSSy9RXl7Oxz72sbQPDpaWlnZvx2Ix9u/f3+/P6W08Nx6Ps3z5cp599lkWLlzID37wA5577jl+9KMfsWzZMp566ilmzJjB6tWrD2gpHSwFyiBEB+bf3KwWioj0GD16NHv2pP+9sGvXLg477DDKy8t54403ePnllzP2c4877jg2bNhAY2Mj06ZN45FHHuHcc8+lubmZffv2cckll3DmmWcybdo0AN566y3OOOMMzjjjDP7rv/6LjRs3KlByIdrl1RjO9NKTuSICUF1dzdlnn82JJ57IqFGjmDhxYvex2bNn86Mf/YiTTz6ZY489ljPPPDNjP7esrIyHHnqIq666qntQ/ktf+hLbt29nzpw5tLS04O7cfffdAHzta19j3bp1uDsXXHABp5xyyiHXwbI57dXMZgPfB2LAg+5+Z8rxUuAnwGnANuAad98QHpsP3AB0AF929yWR82LASmCTu18W7rsA+C7BuFAzcL27N/ZVv5kzZ/pgVmx0d2bc/jt27W8H4E/zzufIsRqYFxkOXn/9dY4//vhcV6NgpPvzNLNV7j4ztWzWBuXDX/r3AhcD04FPm9n0lGI3ADvcfRpwN/Cd8NzpwFzgBGA28MPwel1uBV5PudZ9wGfdfQbwn8C/ZPaOepiZHnAUEUmRzVles4BGd1/v7m3AQmBOSpk5wMPh9iLgAgv6juYAC9291d3fBhrD62FmtcClwIMp13JgTLhdBbyX4ftJUj9RA/MiMnRuvvlmZsyYkfT10EMP5bpaSbI5hjIJ2Bj53ASc0VsZd0+Y2S6gOtz/csq5k8Lte4CvA6NJdiPwtJntB3YDaTsnzewm4CaAKVOmHNwdRTRMUAtFRIbOvffem+sq9CubLZR0o9SpAza9lUm738wuAza7+6o0x/8JuMTda4GHgLvSVcrd73f3me4+s6ampvfa9yN5ppdaKCIi2QyUJmBy5HMtB3ZDdZcxszhBV9X2Ps49G7jczDYQdKGdb2aPmlkNcIq7LwvL/wz4u4zeTYppSTO99uidXiIy4mUzUFYA9WY21cxKCAbZF6eUWQxcF25fCTznwW/mxcBcMys1s6lAPbDc3ee7e62714XXe87drwV2AFVm1hBe60IOHLTPqJrKUsaWB+ss723r4L1dWr9aREa2rI2hhGMitwBLCKYNL3D3NWZ2O7DS3RcDPwYeMbNGgpbJ3PDcNWb2OPAakABudvde3w0Q/qx/BJ4ws06CgPlitu4NwpleE0azfMN2IBhHmaSpwyIygmX1XV7u/rS7N7j7Me5+R7jvW2GY4O4t7n6Vu09z91nuvj5y7h3hece6+2/SXHtp1zMo4ecn3f0kdz/F3T8WvVa21OudXiKSYrDroQDcc8897Nu3r88ydXV1bN26dVDXzza9HPIQJA3Ma+qwiJD9QBnO9OqVQ6AWisgw95t58MHfMnvNw0+Ci+/s9XB0PZQLL7yQCRMm8Pjjj9Pa2soVV1zBt7/9bfbu3cvVV19NU1MTHR0dfPOb3+TDDz/kvffe47zzzmP8+PE8//zz/VblrrvuYsGCBQDceOONfOUrX0l77WuuuSbtmiiZpkA5BEmrN25uprPTKSrSO71ERrLoeijPPPMMixYtYvny5bg7l19+OS+88AJbtmzhyCOP5KmnggXAdu3aRVVVFXfddRfPP/8848eP7/fnrFq1ioceeohly5bh7pxxxhmce+65rF+//oBrb9++Pe2aKJmmQDkE4ytLGVdRwva9bexr62DTzv1MHlee62qJSJc+WhJD4ZlnnuGZZ57h1FNPBaC5uZl169ZxzjnncNttt/GNb3yDyy67jHPOOeegr/3iiy9yxRVXdL8e/5Of/CR//OMfmT179gHXTiQS3WuiXHrppVx22WX9XH1wNIZyiOojT8yv06vsRSTC3Zk/fz6rV69m9erVNDY2csMNN9DQ0MCqVas46aSTmD9/Prfffvugrp1Oumt3rYnyqU99il/+8pfMnj37UG8tLQXKIapPekmkBuZFRrroeigXXXQRCxYsoLk5+N2wadMmNm/ezHvvvUd5eTnXXnstt912G6+88soB5/bn7//+7/nlL3/Jvn372Lt3L08++STnnHNO2ms3Nzeza9cuLrnkEu655x5Wr16dlXtXl9chatBLIkUkIroeysUXX8xnPvMZzjrrLAAqKyt59NFHaWxs5Gtf+xpFRUUUFxdz3333AXDTTTdx8cUXc8QRR/Q7KP+Rj3yE66+/nlmzZgHBoPypp57KkiVLDrj2nj170q6JkmlZXQ9luBvseihRL721jU8/ELzH8uTaKhbf8tFMVE1EBknroWTWsFgPZaRoSJo6HMz0EhEZidTldYiqK0uprihh29429rdrppeIZMYZZ5xBa2tr0r5HHnmEk046KUc16p8CJQPqJ1aybX3PO70UKCK55e4Ea/Xlr2XLlvVfKMsOdkhEXV4ZoFewiAwfZWVlbNu2TUtKHCJ3Z9u2bZSVlQ34HLVQMiB5OWA9iyKSS7W1tTQ1NbFly5ZcVyXvlZWVUVtbO+DyCpQMSFoOWA83iuRUcXExU6dOzXU1RiR1eWVAtIXSuFkzvURkZFKgZMC4ihLGV5YA0NLeycYd+fv6aRGRwVKgZEj9BD0xLyIjW1YDxcxmm9laM2s0s3lpjpea2c/C48vMrC5ybH64f62ZXZRyXszM/mxmv47sMzO7w8zeNLPXzezL2by3VNEHHDWOIiIjUdYG5c0sBtwLXAg0ASvMbLG7vxYpdgOww92nmdlc4DvANWY2nWB9+ROAI4Hfm1lDZF35W4HXgTGRa10PTAaOc/dOM5uQrXtLp17v9BKRES6bLZRZQKO7r3f3NmAhMCelzBzg4XB7EXCBBU8jzQEWunuru78NNIbXw8xqgUuBB1Ou9d+A2929E8DdN2fhnnqV/CyKWigiMvJkM1AmARsjn5vCfWnLuHsC2AVU93PuPcDXgc6Uax1D0LpZaWa/MbP6dJUys5vCMiszOU892uXVuLmZDs30EpERJpuBku69B6m/ZXsrk3a/mV0GbHb3VWmOlwIt4RswHwAWpKuUu9/v7jPdfWZNTU3vtT9IY8tLqBldCkBropON2zXTS0RGlmwGShPBmEaXWuC93sqYWRyoArb3ce7ZwOVmtoGgC+18M3s0cq0nwu0ngZMzdSMDlTQwr24vERlhshkoK4B6M5tqZiUEg+yLU8osBq4Lt68EnvPgBTyLgbnhLLCpQD2w3N3nu3utu9eF13vO3a8Nz/8lcH64fS7wZrZurDdJU4c3a2BeREaWrM3ycveEmd0CLAFiwAJ3X2NmtwMr3X0x8GPgETNrJGiZzA3PXWNmjwOvAQng5sgMr97cCTxmZv8ENAM3ZuXG+lCvFoqIjGBZfZeXuz8NPJ2y71uR7Rbgql7OvQO4o49rLwWWRj7vJJj9lTNaDlhERjI9KZ9BDZEur7e2aKaXiIwsCpQMqiovZkJkpte7muklIiOIAiXD9ICjiIxUCpQMiw7Ma7EtERlJFCgZpuWARWSkUqBkmB5uFJGRSoGSYdMiM73Wb9lLoiP1lWMiIoVJgZJhVaOKmTgmmOnV1tHJO5rpJSIjhAIlC/SAo4iMRAqULEheDljjKCIyMihQsiB5OWC1UERkZFCgZEHycsBqoYjIyKBAyYLow42a6SUiI4UCJQvGlBVzRFUZEMz02rBNM71EpPApULJE3V4iMtIoULKkYUL0iXkNzItI4VOgZEnSO702q4UiIoVPgZIl0/TWYREZYbIaKGY228zWmlmjmc1Lc7zUzH4WHl9mZnWRY/PD/WvN7KKU82Jm9mcz+3Waa/6HmeW8j6k+0uX19ta9tGuml4gUuKwFipnFgHuBi4HpwKfNbHpKsRuAHe4+Dbgb+E547nRgLnACMBv4YXi9LrcCr6f5mTOBsRm+lUEZXVbMkeFMr/YO551te3NcIxGR7MpmC2UW0Oju6929DVgIzEkpMwd4ONxeBFxgZhbuX+jure7+NtAYXg8zqwUuBR6MXigMnO8CX8/S/Ry0eq2NIiIjSDYDZRKwMfK5KdyXtoy7J4BdQHU/595DEBqpfUi3AIvd/f2+KmVmN5nZSjNbuWXLloHfzSBobRQRGUmyGSiWZp8PsEza/WZ2GbDZ3VclXcTsSOAq4D/6q5S73+/uM919Zk1NTX/FD0m93josIiNINgOlCZgc+VwLvNdbGTOLA1XA9j7OPRu43Mw2EHShnW9mjwKnAtOAxvBYuZk1Zvh+DlrycsBqoYhIYctmoKwA6s1sqpmVEAyyL04psxi4Lty+EnjO3T3cPzecBTYVqAeWu/t8d69197rwes+5+7Xu/pS7H+7udeGxfeFAf06lzvRqS2iml4gUrqwFSjgmcguwhGBG1uPuvsbMbjezy8NiPwaqw9bEV4F54blrgMeB14DfAje7e0e26potFaVxJo0dBUCi09mgmV4iUsDi2by4uz8NPJ2y71uR7RaCsY90594B3NHHtZcCS3s5Vplufy7UT6xk0879QNDtFe0GExEpJHpSPsu0HLCIjBQKlCyLjqOs0zu9RKSAKVCyrEEPN4rICKFAybJpkRbKBs30EpECpkDJsorSOLWH9cz0enurZnqJSGFSoAwBPeAoIiOBAmUI1GttFBEZARQoQ6BhggbmRaTwKVCGQLSFouWARaRQKVCGQHSm1zvb9tGayLu3yIiI9EuBMgTKS+JMHhfM9OrQTC8RKVAKlCGicRQRKXQKlCGSvNiWxlFEpPAoUIaIlgMWkUKnQBkieuuwiBQ6BcoQOaamErNge8O2vbS0a6aXiBQWBcoQGVUSY8q4cgA6HdZv0UwvESksWQ0UM5ttZmvNrNHM5qU5XmpmPwuPLzOzusix+eH+tWZ2Ucp5MTP7s5n9OrLvsbDsq2a2wMyKs3lvg1EfmemltVFEpNBkLVDMLAbcC1wMTAc+bWbTU4rdAOxw92nA3cB3wnOnA3OBE4DZwA/D63W5lWCd+qjHgOOAk4BRwI0ZvaEMqNfAvIgUsGy2UGYBje6+3t3bgIXAnJQyc4CHw+1FwAVmZuH+he7e6u5vA43h9TCzWuBS4MHohdz9aQ8By4HaLN3XoDUkvSRSA/MiUliyGSiTgI2Rz03hvrRl3D0B7AKq+zn3HuDrQNqVqsKurs8Bv+3l+E1mttLMVm7ZsuVg7ueQJXd5KVBEpLBkM1AszT4fYJm0+83sMmCzu6/q4+f+EHjB3f+Y7qC73+/uM919Zk1NTR+XybxpEyopCu/sHc30EpECM6BAMbNbzWyMBX5sZq+Y2cf7Oa0JmBz5XAu811sZM4sDVcD2Ps49G7jczDYQdKGdb2aPRur5r0AN8NWB3NdQKytOnun11ha1UkSkcAy0hfJFd98NfJzgF/YXgDv7OWcFUG9mU82shGCQfXFKmcXAdeH2lcBz4RjIYmBuOAtsKlAPLHf3+e5e6+514fWec/drAczsRuAi4NPuPmwXbq/XA44iUqAGGihdXVCXAA+5+19I3y3VLRwTuQVYQjAj63F3X2Nmt5vZ5WGxHwPVZtZI0KqYF567BngceI1gLORmd++vf+hHwETgJTNbbWbfGuC9DSm9gkVEClV8gOVWmdkzwFRgvpmNppdB8Sh3fxp4OmXftyLbLcBVvZx7B3BHH9deCiyNfB7oveRU8vryaqGISOEY6C/hG4AZwHp332dm4wi6veQg6eFGESlUA+3yOgtY6+47zexa4F8IpvjKQTq6pqJ7pte72/exv00zvUSkMAw0UO4D9pnZKQTPgLwD/CRrtSpgZcUxjqquAMA100tECshAAyURzr6aA3zf3b8PjO7nHOlFfWSNeXV7iUihGGig7DGz+QRPoD8Vvldr2L18MV9oYF5ECtFAA+UaoJXgeZQPCF6D8t2s1arA1Se900stFBEpDAMKlDBEHgOqwteftLi7xlAGSS0UESlEA331ytUEb/C9CrgaWGZmV2azYoXs6JoKYuFUr407NNNLRArDQJ9D+WfgdHffDGBmNcDvCV45LwepNB7jqOpy1m/Zizs0bm7mpNqqXFdLROSQDHQMpagrTELbDuJcSaNhQrTbS+MoIpL/BhoKvzWzJWZ2vZldDzxFyitV5OAkvdNLU4dFpAAMqMvL3b9mZp8ieH28Afe7+5NZrVmBm6a3DotIgRnwCxXd/QngiSzWZURJWg5YLRQRKQB9BoqZ7eHAVRYhaKW4u4/JSq1GgKnjg5leHZ3Oxu372deWoLwkL16YLCKSVp9jKO4+2t3HpPkarTA5NKXxGHXV5d2fG7XGvIjkOc3UyiE94CgihUSBkkPJywFrHEVE8psCJYe0HLCIFJKsBoqZzTaztWbWaGbz0hwvNbOfhceXmVld5Nj8cP9aM7so5byYmf3ZzH4d2Tc1vMa68Jol2by3TFCXl4gUkqwFSviK+3uBi4HpwKfNbHpKsRuAHe4+Dbgb+E547nRgLnACMBv4YXi9LrcCr6dc6zvA3e5eD+wIrz2s1VVXEA/f6bVp5372tiZyXCMRkcHLZgtlFtDo7uvdvQ1YSLBAV9Qc4OFwexFwgZlZuH+hu7e6+9tAY3g9zKwWuBR4sOsi4Tnn0/NusYeBT2TlrjKoJF5E3fiK7s/rNNNLRPJYNgNlErAx8rkp3Je2jLsnCNapr+7n3HsIliHujByvBnaG1+jtZwFgZjeZ2UozW7lly5aDvaeMa9DaKCJSILIZKJZmX+pDkr2VSbs/XItls7uvGsTPCna63+/uM919Zk1NTboiQ6o+8pJItVBEJJ9lM1CagMmRz7XAe72VMbM4UAVs7+Pcs4HLzWwDQRfa+Wb2KLAVGBteo7efNSwlD8yrhSIi+SubgbICqA9nX5UQDLIvTimzGLgu3L4SeM7dPdw/N5wFNhWoB5a7+3x3r3X3uvB6z7n7teE5z4fXILzmr7J4bxmT3OWlFoqI5K+sBUo4nnELsIRgRtbj7r7GzG43s8vDYj8Gqs2sEfgqMC88dw3wOPAa8FvgZnfvb1nDbwBfDa9VHV572KsbX0FxrGemV7NmeolInrLgH/cj08yZM33lypW5rgYfv/sP3c+hPPnf/45TpxyW4xqJiPTOzFa5+8zU/XpSfhio19ooIlIAFCjDgJYDFpFCoEAZBpKXA1YLRUTykwJlGKiPBEqjWigikqcUKMPAUdU9M73e29XCnpb2HNdIROTgKVCGgeJYEUePj64xr24vEck/CpRhol7v9BKRPKdAGSa0NoqI5DsFyjCh1RtFJN8pUIYJPdwoIvlOgTJMHDWunJJY8Nfxwe4Wdu3XTC8RyS8KlGEiHivi6Jqe1RsbN6vbS0TyiwJlGKnXwLyI5DEFyjDSMEFro4hI/lKgDCNJA/Pq8hKRPKNAGUY0dVhE8pkCZRg5qrqCknjwV/Lh7lbN9BKRvJLVQDGz2Wa21swazWxemuOlZvaz8PgyM6uLHJsf7l9rZheF+8rMbLmZ/cXM1pjZtyPlLzCzV8xstZm9aGbTsnlv2RArMo6p0StYRCQ/ZS1QzCwG3AtcDEwHPm1m01OK3QDscPdpwN3Ad8JzpwNzgROA2cAPw+u1Aue7+ynADGC2mZ0ZXus+4LPuPgP4T+BfsnVv2ZTc7aWBeRHJH9lsocwCGt19vbu3AQuBOSll5gAPh9uLgAvMzML9C9291d3fBhqBWR7o+i1bHH55+NmBMeF2FfBeNm4q25Lf6aUWiojkj3gWrz0J2Bj53ASc0VsZd0+Y2S6gOtz/csq5k6C75bMKmAbc6+7LwjI3Ak+b2X5gN3AmaZjZTcBNAFOmTBnsvWVNfXTqsGZ6iUgeyWYLxdLs8wGW6fVcd+8Iu7VqgVlmdmJ4/J+AS9y9FngIuCtdpdz9fnef6e4za2pqBnAbQ0sPN4pIvspmoDQBkyOfazmwG6q7jJnFCbqqtg/kXHffCSwlGEepAU6JtFZ+BvxdRu5iiE0ZV05pONNry55Wdu5ry3GNREQGJpuBsgKoN7OpZlZCMMi+OKXMYuC6cPtK4Dl393D/3HAW2FSgHlhuZjVmNhbAzEYB/wC8AewAqsysIbzWhcDrWby3rDlgppdWbxSRPJG1MZRwTOQWYAkQAxa4+xozux1Y6e6LgR8Dj5hZI0HLZG547hozexx4DUgAN7t7h5kdATwcjqMUAY+7+68BzOwfgSfMrJMgYL6YrXvLtoaJlbz2/m4gGJg/vW5cjmskItK/bA7K4+5PA0+n7PtWZLsFuKqXc+8A7kjZ91fg1F7KPwk8eYhVHha0NoqI5CM9KT8MaeqwiOQjBcowpIcbRSQfKVCGocmHlVNWHPzVbG1uZcdezfQSkeFPgTIMFRUZ0ybozcMikl8UKMNUw4TIOIqmDotIHlCgDFPTIuMojWqhiEgeUKAMU0ktFA3Mi0geUKAMUw1aDlhE8owCZZiqPWwUo4pjAGxtbmO7ZnqJyDCnQBmmNNNLRPKNAmUYq5+o5YBFJH8oUIaxBq2NIiJ5RIEyjCW/gkUtFBEZ3hQow1j9hOhML7VQRGR4U6AMY5PG9sz02r63jZsfe4VFq5rY2tya45qJiBwoq+uhyKEpKjKmHzmGVe/sAOCpv73PU397HzM4uXYs5x1bw3nHTuCkSVUUFVmOaysiI50FK+6OTDNnzvSVK1fmuhp9enn9Nm77+V9o2rG/1zLjK0s4t2EC5x83gXMaxjOmrHgIaygiI42ZrXL3mQfsV6AMIlD+8P/C3q1w+o1Q09B/+UPk7qz9cA/Pv7GF59/YzKp3d9DRmf7vLV5knHbUYZx/3ATOO24C9RMqMVPrRUQyJyeBYmazge8TrCn/oLvfmXK8FPgJcBqwDbjG3TeEx+YDNwAdwJfdfYmZlQEvAKUE3XWL3P1fw/IG/F8ESwp3APe5+7/3Vb9BB8rTX4OVD0FnO0w9NwiWYy+B2ND0IO7a184L67bw/NrN/GHtFrb18RT9pLGjOO+4Gs4/bgJnHT2eUSWxIamjiBSuIQ8UM4sBbwIXAk3ACuDT7v5apMx/B0529y+Z2VzgCne/xsymAz8FZgFHAr8HGoBOoMLdm82sGHgRuNXdXzazLwDnAde7e6eZTXD3zX3V8ZC6vJo3wys/CYJldxOMmQSnfQFOuw4qJwzumoPQ2en8pWknz68NWi9/27Sr17Kl8SLOOqaa844NuscmjysfsnqKSOHIRaCcBfybu18Ufp4P4O7/T6TMkrDMS2YWBz4AaoB50bLRcpFzywkC5b+5+zIzWw58xt0bB1rHjIyhdCTgzd/Cigdg/VIoKobpc4JWy5QzYYi7mzbvaWHp2i0sXbuZP765lT2tiV7LHlNTEXSNHTuBmXXjKIlr0p+I9K+3QMlmH80kYGPkcxNwRm9l3D1hZruA6nD/yynnToLuls8qYBpwr7svC8scA1xjZlcAWwi6ydalVsrMbgJuApgyZcqh3F8gFofjLwu+tq6DFT+G1f8Jry6CiScGwXLy1VBSceg/awAmjC7j6pmTuXrmZNo7Olm5YQfPr93M829sPuBZlre27OWtLW/zwB/fprI0zkenjef84yZw7rE1TBxTNiT1FZHCkc1ASfdP89TmUG9lej3X3TuAGWY2FnjSzE5091cJxlVa3H2mmX0SWACcc8BF3O8H7oeghTLQmxmQ8fVw8Z1wwTfhr4/Digfh11+B3/0rzLyv2bEAABJISURBVPgMnH5DUGaIFMeCLq6zjqnm/7zkeDZu38fStZt57o3N/OmtbbQmOrvLNrcm+O2aD/jtmg8AGFdRQl11OVPHVzJ1fPC9bnw5ddUVVJRqtrmIHCibvxmagMmRz7XAe72UaQq7vKqA7QM51913mtlSYDbwanjOE+HhJ4GHMnIXg1FSATO/AKddDxuXwfIHgnBZdh8c/TE4/R+hYfaQDeJ3mTyunM+dVcfnzqqjpb2Dl97axvNhwKROS96+N3hl/ivv7jzgOhPHlFJXXcHU8cFX3fgKjh5fweRx5ZQVa9BfZKTK5hhKnGBQ/gJgE8Gg/GfcfU2kzM3ASZFB+U+6+9VmdgLwn/QMyj8L1APjgPYwTEYBzwDfcfdfm9mdwJvuvsDMPgZ8191P76uOQ/ocyp4Pg0H8VQ/B7k0wpjYInY9cB5U1Q1OHXrg7b21p5rk3gnD587s7k1ovA2UGR1aN4uiaCuqqe4KmbnwFtYeNojimMRqRQpCracOXAPcQTBte4O53mNntwEp3XxxOA34EOJWgZTLX3deH5/4z8EUgAXzF3X9jZicDD4fXKwIed/fbw/JjgceAKUAz8CV3/0tf9cvJg40dCVj7dNBiefsPwSD+CZ8IWi2TZw35IH46nZ3O+7tb2LB1L2+HX13b727fR6KXZ2D6Ei8yJo8rT+pGqwtbOEdUjSKmJ/1F8oYebEwj50/Kb3kzCJa//BRad8PhJwWD+CddNWSD+Acr0dHJpp37WR+GzIate4PtbXvZtGM/g8gazKCyNE7VqGLGlBUzZlQ8/F7c+77I54qSmB7eFBlCCpQ0ch4oXVqb4W+Pw/IHYfMaKK2CUz8LM2+A8dNyXbsBa010sHH7Pt7euo+3tzbz9tZ93S2bD3a3ZO3nxoqMMWVxxqSETxA8xQccqyiJM6okRnlJjLLiGKOKY5SXxCkrLlIwiQyAAiWNYRMoXdzh3ZeCQfzXF0NnAo4+D2b9I9RfNOSD+Jm0ry3BO9v2HdCFtmHbXrY29/6k/1AbVRxjVEms+3tX6JR37YscLy+JUVYSo7xrX0m8Z3/ke7zIKI4VESsyimMWfg8+x4tMISZ5R4GSxrALlKg9H/Q8ib/nPSgfHzwweeInYcpZUFQ4s6k6Op09Le3s3p9gd0s7u/e3s2t/e7idui/B7sixXfvb2d/eketbOCTxouSQ6QqdeFER8VgQOt3bsaLws4XHipK2S4uLGFNWTGVpnNFlcUaXFTO6LE5lWZwx0c+lQUtNb6mWwVCgpDGsA6VLRzu8uSR4UHLtbyGxHyoPhxOuCMKl9vRhMZCfS22JziCQWoKA2Z0SRj37gjDa15Zgf3sH+9o6aGnrYF97B/vbOgY1sy2fdY1dJQdQEDqV4faYSAB1hdHosjijS4PtIjM63Ol0p7PTw+1gYkdH+Nnd6egM/uHQGZbt2Q73d4bbXddJKgud7jhQEgtCszReRGk8Rmm8iLLiYLsknrxfYZk9CpQ08iJQolqbg9e8rHkS1j0DHW1QNSWYJXbip+CIU0Z8uByKjk6npb2D/WHAdIVOsJ1gf1sn+9oStHTtTy3XHgZU5FhLooNEh9Pe0UlHZ+R71y/cwcxikAEpjll3uJTGiygtjmxHA6i4KG25IjM63Ul0elJAdkQCL9iGjs5OOlKCNLVc93Ui4Zvo6DkeLyqiJB5+xSLb8SJKY8nHSouLKInF+iwTPVYSC+4t+RoxKgf5kLICJY28C5Soll3wxtPw6hOw/vlgvGXc0XDCJ4NwmTg91zWUAYj+Ymnv7KSj63tnsC/R6SQ6OsPvPce6gim1zP72Dva0JNjT0s6elgTNLQn2tAbbu1sSNIf797Qk8r6rUA7NmLI4f/23iwZ1bi7e5SXZVFYFMz4dfO3bHgziv/oLePEu+OP3oOa4IFhO+GRezRQbaYqKjCKM4hiMYmjHxdo7OtnbmgjDpidomlt7tne3tAehFA2pyDl4cA+xIqPIjCKjezv4Hh4PP5sZsSKImQX3bhZuEzkncm7Xdth91Z7opDXRSWsi6KJsbY9sJzppbe/Zlr6VxDP/35taKPnaQulN82Z47VdBuLz7p2Df4Sf1hMthR+W2fiJDwN1p7/BewyZpO9FBa3snbR3J+xOd3j1hoqgrCIuKiHWFZiQoY5Fy8ZRjReEkiqLusl3XCYI0FpZNdDptia56dNLW0UFbWJeu/W2JyFf4ubWPY22JTlq7tzuS9o+vLOV3Xz13UH++6vJKoyADJWrXJnjtl0G4bArvc9LMYDD/hCtgzJG5rZ+I5CUFShoFHyhROzYEg/mvPgEf/A2wYPrxiZ8MpiMP4aJgIpLfFChpjKhAidraCGt+EYTLljfAiqDunKBb7Pj/A8rH5bqGmdfRDnveD1ptbXuDGXLRr0RrUKajDToi24m2fspG93dtt4fXCB/YHDMJxk5J/1U2VjPzJO8oUNIYsYES9eFrQbCs+QVsXw9FcTjqbDisDionwuiJwXMv3dsTIV6a61oncw8mJuxugl29fDV/AH6QA7WxkgO/4l3bxRAr7X1frDj4c/LOIMR2vgs734G25EXOKB2TEjJHJX8eNTZzf04iGaJASUOBEuEO768OxlvWLw2e1N+7hQPXRCP4V/Xow4NussrDe4KmMtw3OgygsqrM/Ou7fX/wS3nXxuDV/7uagu1dm3oCI5G8nguxUqiqhapJUDU52B4zKfhcOib9L/+k8CjOfMvBHfbvCIJl57vpvw4InKreWzcKHMkRBUoaCpR+dCSCUGn+sOdrz4fBv/aj23s+DLp4UsXLekInGjSVE5MDCXpCYnckJLpCY9/WA69defiBgdEdGpOhYnz+dSX1Fzg73oH2vcnnpAZO1aSUP+8J6lYrZO7BV/ChZ1/SdngstVysFIoGt0aRnkORgxeLw5gjgq++uAcPWvYVOtsaYcOL0HLgCpAHKKnsCYkjTw3DItLKGHPk8Ot2ywSzYPyqfFxw36n6Cpwdbwcty9TAgeAXR7TLsjvUJyYHfEVN0DKT7GvZBU0rw6/lsOmVoHXaVwBEgyITbl4BNQ2Zux4KFMkEs6DrZdRYqDm277KJ1kjQhGEDyYGRqa6yQjOQwGnZFTyL1PxB8H3PB8ktzG1vwTt/gv3b0/0AKK/uJ3zCVk/paP0dDVRnJ2x9MwiOphWwcUUwGQYHDCYcD8dfFvzZQ7APwj/f6HZ4LLrdZzn6LlcxPoM3GVCgyNCKl/Z0z0hmJQV7P//yTLTB3s3Jwd4dQGEgbV0XHOtIs7xAcXnYnVYFxRXBgnCpX937y4NWZ0lFcF7XdnR/rKRwAmr/zuC5r40rggDZtDIIegi6H2tP73mx66TToGxMbuubQVkNFDObDXyfYMneB939zpTjpcBPgNOAbcA17r4hPDYfuAHoAL7s7kvCJYNfAErDui9y939NueZ/AF9w98ps3ptIXouX9Iw79aWrmy2pOzPy1bonmIa9b2s4qWBv0HXTvi99EPXGYmG4lKcJpPCrdHTyRJDRRwTb5eNyF0adnbB1LWxcHrZAVoatDwhaH9ODh4hrZwVLfI87ZtDjFvkga4FiZjHgXuBCoAlYYWaL3f21SLEbgB3uPs3M5gLfAa4xs+nAXOAE4Ejg92bWALQC57t7s5kVAy+a2W/c/eXwZ84ENO1FJFOi3WwTjj+4cxNtwZhO277koOna7nd/V1C9E2y37Ia2PQf+nFhJ8ljQ6MPD0Dm8ZyLI6COCLqVD/WW+fwc0rQrCY+Ny2LQqWL4bYNRhYevjSpgctj5KRx/az8sz2WyhzAIa3X09gJktBOYA0UCZA/xbuL0I+IEFy9fNARa6eyvwtpk1ArPc/SWga15lcfjl4fVjwHeBzwBXZPG+RGQg4uEzOqMOy9w12/b2jAvt+SDcDmca7nm/78kfRXGomHBg0HS3diKTE4pi0NkRtDa6xj2algdjIRA8DDzhhOBh4MmzghZI9TGF0203SNkMlEnAxsjnJuCM3sq4e8LMdgHV4f6XU86dBN3BsQqYBtzr7svCMrcAi939/b6WVDWzm4CbAKZMUT++SF4pqQh+cVcf03e59v09XXR73g+33++ZdbhjA7z7cvrJCVYUBE/b3p4W0ahxQXCcfHUQHpM+MuJaHwORzUBJ91s9dc5bb2V6PdfdO4AZZjYWeNLMTgS2A1cBH+uvUu5+P3A/BM+h9FdeRPJQ8ajgbQ+H1fVdLmnW4Qc9rZ49HwQTSGpPD8c+jh7xrY+ByGagNAGTI59rgfd6KdNkZnGgiiAc+j3X3Xea2VJgNvA6QYulMWydlJtZo7trIRAR6Z1mHWZUNqcbrADqzWyqmZUQDLIvTimzGLgu3L4SeM6DR/cXA3PNrNTMpgL1wHIzqwlbJpjZKOAfgDfc/Sl3P9zd69y9DtinMBERGVpZa6GEYyK3AEsIpg0vcPc1ZnY7sNLdFwM/Bh4JB923E4QOYbnHCQbwE8DN7t5hZkcAD4fjKEXA4+7+62zdg4iIDJze5aV3eYmIHJTe3uVVuE/YiIjIkFKgiIhIRihQREQkIxQoIiKSEQoUERHJiBE9y8vMtgDv5LoeAzQeSLN0YUHQveWvQr4/3VvvjnL3mtSdIzpQ8omZrUw3Ta8Q6N7yVyHfn+7t4KnLS0REMkKBIiIiGaFAyR/357oCWaR7y1+FfH+6t4OkMRQREckItVBERCQjFCgiIpIRCpRhzMwmm9nzZva6ma0xs1tzXadMM7OYmf3ZzApuGQIzG2tmi8zsjfDv8Kxc1ylTzOyfwv8mXzWzn5pZWa7rdCjMbIGZbTazVyP7xpnZ78xsXfj9sFzWcbB6ubfvhv9d/tXMnuxaZ+pQKVCGtwTwP939eOBM4GYzm57jOmXarQQrbhai7wO/dffjgFMokPs0s0nAl4GZ7n4iwXpHc3Nbq0P2vwlWf42aBzzr7vXAs+HnfPS/OfDefgec6O4nA28C8zPxgxQow5i7v+/ur4Tbewh+IU3Kba0yx8xqgUuBB3Ndl0wzszHA3xMsIoe7t7n7ztzWKqPiwKhw6e5yDlzeO6+4+wsEi/xFzQEeDrcfBj4xpJXKkHT35u7PuHsi/PgywTLrh0yBkifMrA44FViW25pk1D3A14HOXFckC44GtgAPhV16D5pZRa4rlQnuvgn4HvAu8D6wy92fyW2tsmKiu78PwT/ugAk5rk+2fBH4TSYupEDJA2ZWCTwBfMXdd+e6PplgZpcBm919Va7rkiVx4CPAfe5+KrCX/O0ySRKOJcwBpgJHAhVmdm1uayWDYWb/TNC1/lgmrqdAGebMrJggTB5z91/kuj4ZdDZwuZltABYC55vZo7mtUkY1AU3u3tWiXEQQMIXgH4C33X2Lu7cDvwD+Lsd1yoYPzewIgPD75hzXJ6PM7DrgMuCznqEHEhUow5iZGUEf/Ovufleu65NJ7j7f3WvdvY5gQPc5dy+Yf+W6+wfARjM7Ntx1AfBaDquUSe8CZ5pZefjf6AUUyISDFIuB68Lt64Bf5bAuGWVms4FvAJe7+75MXVeBMrydDXyO4F/vq8OvS3JdKRmw/wE8ZmZ/BWYA/3eO65MRYatrEfAK8DeC3yN5/ZoSM/sp8BJwrJk1mdkNwJ3AhWa2Drgw/Jx3erm3HwCjgd+Fv1d+lJGfpVeviIhIJqiFIiIiGaFAERGRjFCgiIhIRihQREQkIxQoIiKSEQoUkTxlZh8rxLc0S/5SoIiISEYoUESyzMyuNbPl4QNk/ytcA6bZzP4/M3vFzJ41s5qw7AwzezmyTsVh4f5pZvZ7M/tLeM4x4eUrI2uuPBY+uS6SEwoUkSwys+OBa4Cz3X0G0AF8FqgAXnH3jwB/AP41POUnwDfCdSr+Ftn/GHCvu59C8N6s98P9pwJfAaYTvOH47KzflEgv4rmugEiBuwA4DVgRNh5GEbxksBP4WVjmUeAXZlYFjHX3P4T7HwZ+bmajgUnu/iSAu7cAhNdb7u5N4efVQB3wYvZvS+RAChSR7DLgYXdPWhHPzL6ZUq6vdyD11Y3VGtnuQP9PSw6py0sku54FrjSzCdC9TvlRBP/vXRmW+QzworvvAnaY2Tnh/s8BfwjXwGkys0+E1yg1s/IhvQuRAdC/ZkSyyN1fM7N/AZ4xsyKgHbiZYMGtE8xsFbCLYJwFgtek/ygMjPXAF8L9nwP+l5ndHl7jqiG8DZEB0duGRXLAzJrdvTLX9RDJJHV5iYhIRqiFIiIiGaEWioiIZIQCRUREMkKBIiIiGaFAERGRjFCgiIhIRvz/xxI2bWss2t0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens = 784, 10, 256\n",
    "\n",
    "W1 = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_hiddens)), dtype=torch.float)\n",
    "b1 = torch.zeros(num_hiddens,dtype=torch.float)\n",
    "W2 = torch.tensor(np.random.normal(0, 0.01, (num_hiddens, num_outputs)), dtype=torch.float)\n",
    "b2 = torch.zeros(num_outputs,dtype=torch.float)\n",
    "\n",
    "params = [W1, b1, W2, b2]\n",
    "for param in params:\n",
    "    param.requires_grad = True\n",
    "\n",
    "\n",
    "def relu(X):\n",
    "    return torch.max(input = X, other=torch.tensor(0.0))\n",
    "\n",
    "\n",
    "def net(X):\n",
    "    X = X.view(-1, num_inputs)\n",
    "    H = relu(torch.matmul(X, W1) + b1)\n",
    "    return torch.matmul(H, W2) + b2\n",
    "\n",
    "\n",
    "def sgd(params, lr, batch_size):\n",
    "    for param in params:\n",
    "        param.data -= lr * param.grad / batch_size\n",
    "\n",
    "\n",
    "def evaluate_accuracy(data_iter, net):\n",
    "    acc_sum = 0.0\n",
    "    n = 0\n",
    "    for X, y in data_iter:\n",
    "        acc_sum += (net(X).argmax(dim = 1) == y).float().sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "\n",
    "def l2_penalty(W1, W2):  # 定义L2范数惩罚项\n",
    "    return (W1 ** 2).sum() / 2 + (W2 ** 2).sum() / 2\n",
    "\n",
    "\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "num_epochs, lr = 12, 50.0\n",
    "lambd = 0.02    # 定义超参数lambd\n",
    "\n",
    "\n",
    "def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, params , lr, lambd):\n",
    "    train_ls, test_ls, x_epoch = [], [], []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum = 0.0\n",
    "        train_1_test_sum = 0.0\n",
    "        train_acc_sum = 0.0\n",
    "        n = 0\n",
    "        n_test = 0\n",
    "        for X, y in train_iter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y) + lambd * l2_penalty(W1, W2)    #添加惩罚项\n",
    "            l = l.sum()\n",
    "            if params[0].grad is not None:\n",
    "                for param in params:\n",
    "                    param.grad.data.zero_()\n",
    "\n",
    "            l.backward()\n",
    "            sgd(params, lr, batch_size)\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim =1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in test_iter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test) + lambd * l2_penalty(W1, W2)\n",
    "            l = l.sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "    plt.plot(x_epoch, train_ls, label=\"train_loss\", linewidth=3)\n",
    "    plt.plot(x_epoch, test_ls, label=\"test_loss\", linewidth=1.5)\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, params, lr, lambd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, loss 0.0063, train acc 0.637, test acc 0.819\n",
      "epoch 2, loss 0.0032, train acc 0.835, test acc 0.863\n",
      "epoch 3, loss 0.0026, train acc 0.863, test acc 0.881\n",
      "epoch 4, loss 0.0023, train acc 0.877, test acc 0.891\n",
      "epoch 5, loss 0.0021, train acc 0.886, test acc 0.898\n",
      "epoch 6, loss 0.0020, train acc 0.892, test acc 0.904\n",
      "epoch 7, loss 0.0019, train acc 0.897, test acc 0.907\n",
      "epoch 8, loss 0.0019, train acc 0.900, test acc 0.908\n",
      "epoch 9, loss 0.0018, train acc 0.903, test acc 0.912\n",
      "epoch 10, loss 0.0018, train acc 0.905, test acc 0.913\n",
      "epoch 11, loss 0.0017, train acc 0.907, test acc 0.915\n",
      "epoch 12, loss 0.0017, train acc 0.910, test acc 0.916\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXzV9Z3v8dcnJ/sKhLAlKCCIIgIqgrh1cVRQR9S60NZWW1vajk7t3LFTvV1myp32Ye/ttLZTl3GqVq1Ta7Uqt9pK3cZ6RRAsqKhIRJSwhkBC9uQkn/vH75dwCAnZzuFkeT8fjzzO7/zW788l7/y+39/3+zV3R0REpKdSkl0AEREZXBQcIiLSKwoOERHpFQWHiIj0ioJDRER6JTXZBTgSRo8e7ZMmTUp2MUREBo21a9fucfeizrYNi+CYNGkSa9asSXYxREQGDTP7sKttqqoSEZFeUXCIiEivKDhERKRXhkUbh4gMPc3NzZSVldHQ0JDsogxqmZmZlJSUkJaW1uNjFBwiMiiVlZWRl5fHpEmTMLNkF2dQcncqKiooKytj8uTJPT5OVVUiMig1NDRQWFio0OgHM6OwsLDXT20KDhEZtBQa/deXf4YKji68ta2Kz9+7mv/5+JvJLoqIyICiNo4upEaMl94rZ+KorGQXRURkQNETRxemFuWSlRZh6956Kmoak10cERlgKisrueOOO3p93AUXXEBlZWWvj7v22mt59NFHe31cIig4upAaSeHE4gIA3thWleTSiMhA01VwtLS0HPa4p59+mhEjRiSqWEeEqqoOY1ZJAau37GX91ko+MX1MsosjIl2YdPNTCTnvllsv7HLbzTffzPvvv8+cOXNIS0sjNzeX8ePHs27dOt5++20uueQStm7dSkNDAzfeeCNLly4NyhqOnVdTU8OiRYs488wzeeWVVyguLubJJ58kK6v76vHnnnuOm266iWg0yqmnnsqdd95JRkYGN998M8uXLyc1NZXzzjuPH//4x/zud7/j+9//PpFIhIKCAl566aV+/3NRcBzG7InBXwVvlOmJQ0QOduutt/LWW2+xbt06XnzxRS688ELeeuut9v4Q9957L6NGjaK+vp5TTz2VT33qUxQWFh50jk2bNvGb3/yG//zP/+TKK6/kscce4+qrrz7sdRsaGrj22mt57rnnOPbYY/n85z/PnXfeyec//3kef/xx3n33XcysvTps2bJlPPPMMxQXF/epiqwzCo7DmBMGx/qtlbi7Xv0TGaAO92RwpMybN++gTnQ///nPefzxxwHYunUrmzZtOiQ4Jk+ezJw5cwA45ZRT2LJlS7fX2bhxI5MnT+bYY48F4JprruH222/nhhtuIDMzky996UtceOGFXHTRRQCcccYZXHvttVx55ZVcdtll8bhVtXEcTsnILEZmp1FR20TZvvpkF0dEBrCcnJz25RdffJFnn32WlStXsn79ek466aROO9llZGS0L0ciEaLRaLfXcfdO16emprJ69Wo+9alP8cQTT7Bw4UIA7rrrLv71X/+VrVu3MmfOHCoqKnp7a4dQcByGmam6SkQ6lZeXR3V1dafbqqqqGDlyJNnZ2bz77ru8+uqrcbvucccdx5YtWygtLQXgwQcf5GMf+xg1NTVUVVVxwQUXcNttt7Fu3ToA3n//febPn8+yZcsYPXo0W7du7XcZVFXVjVklI3hxYznryyq5cNb4ZBdHRAaIwsJCzjjjDGbOnElWVhZjx45t37Zw4ULuuusuZs2axfTp0znttNPidt3MzEzuu+8+rrjiivbG8a9+9avs3buXxYsX09DQgLvz05/+FIBvfvObbNq0CXfnnHPOYfbs2f0ug3X12BMPZrYQ+BkQAX7p7rd22J4BPACcAlQAV7n7lnDbLcB1QAvwdXd/Jlw/AvglMBNw4IvuvvJw5Zg7d673dQbA59/dxRd/tYb5k0fx268s6NM5RCT+3nnnHY4//vhkF2NI6OyfpZmtdfe5ne2fsKoqM4sAtwOLgBnAp81sRofdrgP2uftU4KfAj8JjZwBLgBOAhcAd4fkgCKI/uftxwGzgnUTdAwRPHABvbquipTVxISsiMlgkso1jHlDq7pvdvQl4GFjcYZ/FwP3h8qPAORa8urQYeNjdG939A6AUmGdm+cDZwD0A7t7k7vF5v6wLo3MzKB6RRV1TC6W7axJ5KRERrr/+eubMmXPQz3333ZfsYh0kkW0cxUBsK0wZML+rfdw9amZVQGG4/tUOxxYD9UA5cJ+ZzQbWAje6e23Hi5vZUmApwFFHHdWvG5kzcQTbKutZX1bJ9HF5/TqXiMjh3H777ckuQrcS+cTRWaeHjnU9Xe3T1fpU4GTgTnc/CagFbu7s4u5+t7vPdfe5RUVFPS91J2aVBEOPrN+a0IcbEZFBIZHBUQZMjPleAmzvah8zSwUKgL2HObYMKHP3VeH6RwmCJKH0Sq6IyAGJDI7XgGlmNtnM0gkau5d32Gc5cE24fDnwvAeveS0HlphZhplNBqYBq919J7DVzKaHx5wDvJ3AewDgxOICUgze2bGfhubDD2AmIjLUJayNI2yzuAF4huB13HvdfYOZLQPWuPtygkbuB82slOBJY0l47AYze4QgFKLA9e7e9hv774GHwjDaDHwhUffQJicjlaljcnlvVw1v79jPyUeNTPQlRUQGrIT2HHf3p939WHc/xt1/EK77XhgauHuDu1/h7lPdfZ67b4459gfhcdPd/Y8x69eFbRez3P0Sd9+XyHtoMzt8LfcNtXOICH2fjwPgtttuo66u7rD7TJo0iT179vTp/ImmIUd6qK2dY73aOUSExAfHQKYhR3qo7YljfZmeOEQGnD/eDDvfjO85x50Ii27tcnPsfBznnnsuY8aM4ZFHHqGxsZFLL72U73//+9TW1nLllVdSVlZGS0sL3/3ud9m1axfbt2/nE5/4BKNHj+aFF17otig/+clPuPfeewH40pe+xDe+8Y1Oz33VVVd1OidHvCk4emj6uDzSU1PYXF5LVX0zBVlpyS6SiCRR7HwcK1as4NFHH2X16tW4OxdffDEvvfQS5eXlTJgwgaeeCiaaqqqqoqCggJ/85Ce88MILjB49utvrrF27lvvuu49Vq1bh7syfP5+PfexjbN68+ZBz7927t9M5OeJNwdFD6akpzBifz7qtlbxZVsWZ07r/Fy4iR8hhngyOhBUrVrBixQpOOukkAGpqati0aRNnnXUWN910E9/61re46KKLOOuss3p97pdffplLL720fdj2yy67jL/85S8sXLjwkHNHo9FO5+SIN7Vx9EL7xE6qrhKRGO7OLbfcwrp161i3bh2lpaVcd911HHvssaxdu5YTTzyRW265hWXLlvXp3J3p7NxdzckRbwqOXpg9UT3IRSQQOx/H+eefz7333ktNTTCe3bZt29i9ezfbt28nOzubq6++mptuuonXX3/9kGO7c/bZZ/PEE09QV1dHbW0tjz/+OGeddVan5+5qTo54U1VVL8xSA7mIhGLn41i0aBGf+cxnWLAgmHohNzeXX//615SWlvLNb36TlJQU0tLSuPPOOwFYunQpixYtYvz48d02jp988slce+21zJs3Dwgax0866SSeeeaZQ85dXV3d6Zwc8ZbQ+TgGiv7MxxGrtdWZvWwF1Q1RXr3lHMYVZMahdCLSF5qPI34GzHwcQ1FKih0Y8FBPHSIyTKmqqpdml4zg/5VW8EZZJeefMC7ZxRGRQW7+/Pk0NjYetO7BBx/kxBNPTFKJuqfg6KX2HuRb1YNcJNncnWDut8Fr1apV3e+UQH1prlBVVS/F9iBv1VSyIkmTmZlJRUVFn37xScDdqaioIDOzd+21euLopXEFmYzNz2DX/ka2VNQypSg32UUSGZZKSkooKyujvLw82UUZ1DIzMykpKenVMQqOPphVMoI/v72L9WWVCg6RJElLS2Py5MnJLsawpKqqPpijdg4RGcYUHH2gkXJFZDhTcPTBiWFfjg3b99MUbU1yaUREjiwFRx8UZKUxZXQOTdFW3tvVs/FmRESGCgVHH7X1IF+nAQ9FZJhRcPRRW0fAN9TOISLDjIKjj9SDXESGKwVHH80Yn09qirFpdzW1jdFkF0dE5IhRcPRRZlqE48bn0erw1jY9dYjI8KHg6Af15xCR4UjB0Q8HgkNPHCIyfCg4+uFAA7meOERk+FBw9MPUMblkp0co21dPRU1j9weIiAwBCo5+iKQYM4uDjoBvqLpKRIYJBUc/tY2Uqx7kIjJcKDj6qW3oEfUgF5HhQsHRT7FvVmkKSxEZDhQc/VQyMotROensrW2ibF99sosjIpJwCo5+MjNmh9VV6ggoIsNBQoPDzBaa2UYzKzWzmzvZnmFmvw23rzKzSTHbbgnXbzSz82PWbzGzN81snZmtSWT5e0r9OURkOElN1InNLALcDpwLlAGvmdlyd387ZrfrgH3uPtXMlgA/Aq4ysxnAEuAEYALwrJkd6+4t4XGfcPc9iSp7b7W3c2ikXBEZBhL5xDEPKHX3ze7eBDwMLO6wz2Lg/nD5UeAcM7Nw/cPu3ujuHwCl4fkGpLY3q97cVkW0RVPJisjQlsjgKAa2xnwvC9d1uo+7R4EqoLCbYx1YYWZrzWxpVxc3s6VmtsbM1pSXl/frRrpTmJtBycgs6ptbKC2vSei1RESSLZHBYZ2s6/i+alf7HO7YM9z9ZGARcL2Znd3Zxd39bnef6+5zi4qKelrmPmufEVDVVSIyxCUyOMqAiTHfS4DtXe1jZqlAAbD3cMe6e9vnbuBxBkgV1pywnWOd3qwSkSEukcHxGjDNzCabWTpBY/fyDvssB64Jly8HnvegF91yYEn41tVkYBqw2sxyzCwPwMxygPOAtxJ4Dz3W1s6hN6tEZKhL2FtV7h41sxuAZ4AIcK+7bzCzZcAad18O3AM8aGalBE8aS8JjN5jZI8DbQBS43t1bzGws8HjQfk4q8F/u/qdE3UNvzCwuIMVg485qGppbyEyLJLtIIiIJkbDgAHD3p4GnO6z7XsxyA3BFF8f+APhBh3WbgdnxL2n/5WSkMm1MHht3VbNh+35OOXpksoskIpIQ6jkeR7MnasBDERn6FBxxpB7kIjIcKDjiSHOQi8hwoOCIo+nj8khPTeGDPbVU1TUnuzgiIgmh4IijtEgKMyfkA/DGNlVXicjQpOCIs1lhdZXmIBeRoUrBEWeag1xEhjoFR5ypB7mIDHUKjjibVJhDfmYqu6sb2VnVkOziiIjEnYIjzlJSrL0/h6qrRGQoUnAkQFt1lXqQi8hQpOBIgAMdARUcIjL0KDgSIHZSp9bWjnNXiYgMbgqOBBibn8m4/EyqG6N8UFGb7OKIiMSVgiNB2kbK1Wu5IjLUKDgSRD3IRWSoUnAkiHqQi8hQpeBIkJnFQVXV29v30xRtTXJpRETiR8GRIAVZaUwpyqGppZWNO6uTXRwRkbhRcCTQnLCdY536c4jIEKLgSKD2HuRq5xCRIUTBkUDtc5DriUNEhhAFRwIdPz6f1BRj0+4aahqjyS6OiEhcKDgSKDMtwvHj83GHt7apP4eIDA0KjgRTD3IRGWoUHAk2SyPlisgQo+BIsLYe5Ou3qqpKRIYGBUeCHVOUS056hG2V9eypaUx2cURE+k3BkWCRFGsffkQzAorIUKDgOAIODHio6ioRGfwUHEdAewO53qwSkSFAwXEEtL2S+0ZZJe6aSlZEBjcFxxFQPCKL0bnp7KtrZuve+mQXR0SkX3oUHGZ2o5nlW+AeM3vdzM5LdOGGCjNTfw4RGTJ6+sTxRXffD5wHFAFfAG7t7iAzW2hmG82s1Mxu7mR7hpn9Nty+yswmxWy7JVy/0czO73BcxMz+amZ/6GH5k2622jlEZIjoaXBY+HkBcJ+7r49Z1/kBZhHgdmARMAP4tJnN6LDbdcA+d58K/BT4UXjsDGAJcAKwELgjPF+bG4F3elj2AWFW29AjeuIQkUGup8Gx1sxWEATHM2aWB3Q3H+o8oNTdN7t7E/AwsLjDPouB+8PlR4FzzMzC9Q+7e6O7fwCUhufDzEqAC4Ff9rDsA0LbE8db2/YTbdFUsiIyePU0OK4DbgZOdfc6II2guupwioGtMd/LwnWd7uPuUaAKKOzm2NuAf6Kb4DKzpWa2xszWlJeXd1PUxBuVk85Ro7Kpb25h0+6aZBdHRKTPehocC4CN7l5pZlcD3yH4JX84nVVldXwXtat9Ol1vZhcBu919bXcFdve73X2uu88tKirqbvcjon1GQFVXicgg1tPguBOoM7PZBH/tfwg80M0xZcDEmO8lwPau9jGzVKAA2HuYY88ALjazLQRVX580s1/38B6STj3IRWQo6GlwRD3oubYY+Jm7/wzI6+aY14BpZjbZzNIJGruXd9hnOXBNuHw58Hx4neXAkvCtq8nANGC1u9/i7iXuPik83/PufnUP7yHp1INcRIaC1B7uV21mtwCfA84K33BKO9wB7h41sxuAZ4AIcK+7bzCzZcAad18O3AM8aGalBE8aS8JjN5jZI8DbQBS43t1b+nB/A8rM4nxSDDbuqqahuYXMtEj3B4mIDDDWkyEwzGwc8BngNXf/i5kdBXzc3burrhoQ5s6d62vWrEl2MQBYeNtLvLuzmse+toBTjh6V7OKIiHTKzNa6+9zOtvWoqsrddwIPAQVhA3XDYAmNgeZAR0C1c4jI4NTTIUeuBFYDVwBXAqvM7PJEFmyomj1RQ4+IyODW0zaObxP04dgNYGZFwLMEnfakF9peyVUDuYgMVj19qyqlLTRCFb04VmJMH5dHRmoKWyrqqKxrSnZxRER6rae//P9kZs+Y2bVmdi3wFPB04oo1dKVFUmKmklU7h4gMPj1tHP8mcDcwC5gN3O3u30pkwYYy9SAXkcGsp20cuPtjwGMJLMuwoR7kIjKYHTY4zKyaQ8eXgmAsKXf3/ISUaoibHTOpk7sTDAgsIjI4HDY43L27YUWkD44uzKYgK43y6kZ27m9gfEFWsoskItJjejPqcHa8AbV74n7aYCpZvZYrIoOTgqMr9fvgvkXw1D8m5PRt1VVq5xCRwUbB0ZWskXDWP8LbT8Bb8X8noK0Hud6sEpHBRsFxOKd/HYpPCZ46qnfF9dSzw6qqN8uqaG3tfqBJEZGBQsFxOJFUuOQuaKqDP/wD9GAk4Z4ak5/J+IJMqhujbN5TG7fziogkmoKjO0XHwjnfhY1PwRuPxPXUszWxk4gMQgqOnjjt72DiafDHb8L+jrPf9t2sieGbVWrnEJFBRMHREykRuOQOiDbB/70xblVWc9o7AurNKhEZPBQcPVV4DPzNv8CmFbDuobiccmZJAWbwzvb9NEVb43JOEZFEU3D0xrylcPSZ8KdboKqs36fLz0xjyugcmlpaeXfn/jgUUEQk8RQcvZGSAot/Aa0t8OQNcamyap8RUA3kIjJIKDh6a9RkOG8ZbH4B1t7X79OpB7mIDDYKjr6Yex1M+Tg88x3Yt6Vfp1IPchEZbBQcfWEGF/8CLCWosmrte8P28ePzSIsYpeU11DRG41hIEZHEUHD01YiJsPCHsOUv8Nov+3yajNQIx4/Pxz0YfkREZKBTcPTHSZ+DqefCs/8MFe/3+TSxEzuJiAx0Co7+MIOLfw4pafDk9cHbVn2guTlEZDBRcPRX/gRY9CP4aCWsuqtPp5jT3kCuqioRGfgUHPEwewkcuwieWwZ7NvX68ClFueRmpLKtsp7y6sYEFFBEJH4UHPFgBn/7M0jLgse/2usqq0iKMbM4H4C/frQvESUUEYkbBUe85I2FC34M29bAKz/v9eHzJhcC8J0n3tLwIyIyoCk44mnmp+D4i+GFH8Lud3p16NKzp7BgSiG7qxu58q6VrP1wb4IKKSLSPwqOeDKDC38CGXlBlVVLc48Pzc1I5b4vnMp5M8ayvyHKZ3+5ihc27k5gYUVE+kbBEW+5RXDRT2HHOnj5tl4dmpkW4Y7PnswVp5TQ0NzKl+9fw5PrtiWooCIifZPQ4DCzhWa20cxKzezmTrZnmNlvw+2rzGxSzLZbwvUbzez8cF2mma02s/VmtsHMvp/I8vfZjMVBtdV//wh2vtmrQ1MjKfzvy2fxlbOnEG11vvHbdTywcktCiiki0hcJCw4ziwC3A4uAGcCnzWxGh92uA/a5+1Tgp8CPwmNnAEuAE4CFwB3h+RqBT7r7bGAOsNDMTkvUPfTLBT+GrJHw+NeCmQN7wcy45YLjuXnRcbjD957cwG3PvofHaeZBEZH+SOQTxzyg1N03u3sT8DCwuMM+i4H7w+VHgXPMzML1D7t7o7t/AJQC8zxQE+6fFv4MzN+m2aPgb2+DXW/CX37cp1N89WPHcOtlJ5JicNuzm/iX5RtobR2Ytysiw0cig6MY2BrzvSxc1+k+7h4FqoDCwx1rZhEzWwfsBv7s7qs6u7iZLTWzNWa2pry8PA630wfHXQizlsBLP4bt6/p0iiXzjuKOz55MeiSF+1d+yDd+u47mFk0zKyLJk8jgsE7Wdfxzuat9ujzW3VvcfQ5QAswzs5mdXdzd73b3ue4+t6ioqBfFjrNFt0LumOAtq2jfeoUvnDmeX33hVHLSIyxfv50vP7CG+qa+jYslItJfiQyOMmBizPcSYHtX+5hZKlAA7O3Jse5eCbxI0AYycGWNhIv/HcrfgRdv7fNpTp86mt8sPY1ROem8uLGcq+9ZRVVdz1/3FRGJl0QGx2vANDObbGbpBI3dyzvssxy4Jly+HHjegxbg5cCS8K2rycA0YLWZFZnZCAAzywL+Bng3gfcQH9PODYZg/3+3QdmaPp9mVskIHvnKAiYUZLL2w31c+R8r2bW/IY4FFRHpXsKCI2yzuAF4BngHeMTdN5jZMjO7ONztHqDQzEqB/wHcHB67AXgEeBv4E3C9u7cA44EXzOwNgmD6s7v/IVH3EFfn/wDyJsATX4Pm+j6fZuqYXB792ukcU5TDxl3VXH7XK2zZUxvHgoqIHJ4Nh1c8586d62vW9P0v/bh5/3l48FJYcEMQJP2wt7aJL9y3mvVlVYzOzeCBL85jxoT8OBVURIY7M1vr7nM726ae40fSMZ+EuV+ElbfDR6/261SjctJ56MunccbUQvbUNHLV3StZ/YHGtxKRxFNwHGnnLgvmK3/ia9BU169T5Wakcu+1p7Jo5jiqG6J87p5VPPfOrjgVVESkcwqOIy0jDxbfAXs3w3P9HzElIzXCLz5zMp+eN5HGaCtLH1zL718vi0NBRUQ6p+BIhslnwbyvBFPNbnm536eLpBg/vPRE/u7jx9DS6vyPR9Zz78sfxKGgIiKHUnAky9/8M4yaAk/8HTTWdL9/N8yMf1p4HN++4HgAlv3hbf5txUaNbyUicafgSJb0nKDKqvIj+PP34nbaL589hf9z+SwiKca/P1/Kd554ixaNbyUicaTgSKajF8CC62HNPfD+C3E77RVzJ3LnZ08mPTWFh1Z9xNcf/itNUY1vJSLxoeBItk9+BwqnwZM3QEP85ho/74RxPPDFeeRmpPLUGzu47v7XqG2Mxu38IjJ8KTiSLS0LLr0LqrfDim/H9dSnTSnk4aWnUZiTzl827eGzv1zFvtrezQ0iItKRgmMgKJkLZ9wIrz8Aj34xaPeIk5nFBfzuqwsoHpHFuq2VXPkfK9lZpfGtRKTvFBwDxSe+DWf/E7z7NPz7XHj2+3GruppSlMtjXzudaWNy2bS7hk/d+Qqby/v/JpeIDE8KjoEikgaf/Db8/Ro44RJ4+Sfw7yfD2l9Ba//n3hhXkMnvvrqAORNHsK2ynivuWslb26r6X24RGXYUHANNQQlcdjd8+XkonAr/90a466y4vHU1Ijudh740n7OmjaaitonL7nyFf3xkPW+WKUBEpOc0Ou5A5g5vPxn086j8EKadD+f9Lyia3q/TNkVb+fbjb/Lo62W0/es/+agRXHP6JBbNHE96qv6eEBnuDjc6roJjMIg2BsOTvPRjaKqFU6+Dj90MOYX9Ou2HFbU8sPJDHlmzleqG4FXdorwMPjPvKD47/yjG5GfGo/QiMggpOAZ7cLSp3QMv/BDW3gfpefCxb8K8pZCa0a/T1jVFefyv27j/lS28tytoNE+LGItmjuea0ydx8lEjMOtsGngRGaoUHEMlONrsfgdWfBdK/wwjJwdDtR//t9DPX+7uzsrNFdz/yhb+/PYu2kYqObG4gGtOn8RFs8aTmRaJww2IyECn4BhqwdGm9Fl45jtQ/g4cfUYwq+CEk+Jy6rJ9dfz61Y94+LWPqKxrBoLJo5acOpGrTzuaCSOy4nIdERmYFBxDNTgAWqLw1wfg+R9A3R6Y/Wk453uQPyEup29obmH5+u3c/8oWNmwP+pVEUozzZozlmtMnMX/yKFVjiQxBCo6hHBxtGqrgL/8Gr94JFgl6op/x9WAU3jhwd9Z+uI9fvbKFP721k2hYj3XcuDyuOX0Sl8wpJitd1VgiQ4WCYzgER5t9W+DZf4ENj0Pe+ODpY9YSSInfK7a79jfw0Ksf8l+rP2JPTTD2VX5mKledOpHPL5jExFHZcbuWiCSHgmM4BUebj16FP90C21+H8bPh/B/CpDPjeonGaAt/fHMnv3plC+u2VgJB+/w5x43hmtMncebU0arGEhmkFBzDMTgAWlvhrceCJ5D9ZXDcRcEbWIXHxP1S67dWcv8rW/jDGztoagnm/jimKIdrTp/EZSeXkJuRGvdrikjiKDiGa3C0aaqDlbfDyz+FliaY/xU4+ybIGhn3S+2paeQ3qz7i16s+ZNf+RgByM1K5/JQSLj2pmBMm5JMaUc90kYFOwTHcg6NN9U54/l/hr78OQuOMr8OMS2DU5LhfqrmllRUbdnH/K1tYvWVv+/q8jFTmTR7FaVMKWXBMIcePzyeSouoskYFGwaHgONiON4JJoz54Kfg+5gQ47gI47kIYP6ffHQk72rC9it+s/oiXN+1hS0XdQdvyM1OZP6WQBWGQTB+bR4qCRCTpFBwKjs7t/QA2Pg3vPgUfrQRvhfximH5BECRHnwmp6XG95PbKel7dXMHK9ytYubmCsn31B20fmZ3G/MlBiCw4ppBpY3LVwC6SBAoOBUf3aivgvT8FQVL6HETrIaMApp0bhMjUcyEzP+6X3bq3jpWbK3h1cwCfRfAAABE1SURBVAWvvl/B9g6zE47OTWf+lMKgamtKIccU5ShIRI4ABYeCo3ea6mDzi7DxKdj4R6irgJQ0mHx2UJ01/QLIHx/3y7o7H+2ta38aWfl+BburGw/aZ0xeRnv7yIIphRxdmK0gEUkABYeCo+9aW2Dranj3D0GV1r4PgvUTTg5C5LgLoei4uLeLQBAkH+ypbQ+RVzdXtHc4bDO+ILP9aWTBMYXqfCgSJwoOBUd8uEP5u0GAbHwatq0N1o+aEraLXAQT50FKYoYecXdKd9ccFCT7wgEY2xSPyGLBMYXMnzyKEyYUcMyYHDJSNRSKSG8pOBQcibF/e1CV9e5TwRtarc2QPRqmL4TpF8Ixn4C0xI2i29rqvLe7Oqjaer+CVR/spar+4CCJpBiTCrOZPi6PaWPymD4uj2PH5jGpMFv9SUQOQ8Gh4Ei8hv3B/CDvPg2bVkDjfkjLhmM+GVRnTTu/3zMWdqel1Xlnx35e3VzBmi37eG9XNVsqatvnFYmVHklhSlFOe5BMHxuESvGILL0OLIKCQ8FxpEWb4MOXgxB59ymo3g6WEvQXGT8rGDtr3CwYNxMy8hJalIbmFkp317BpdzUbd9bw3q5q3ttVfchrwG2y0yNMG5MbhMm4PKaFoTI2P0ON8DKsJC04zGwh8DMgAvzS3W/tsD0DeAA4BagArnL3LeG2W4DrgBbg6+7+jJlNDPcfB7QCd7v7z7orh4Ijidxhx7qgSmvbWtixHmrLw40WjJs1LgyT8bNg3OyEP5kA1DRG2RSGyMadbcFSfchbXG3yM1MPCpJjx+Zx7NhcCnP7N22vyECVlOAwswjwHnAuUAa8Bnza3d+O2efvgFnu/lUzWwJc6u5XmdkM4DfAPGAC8CxwLDAGGO/ur5tZHrAWuCT2nJ1RcAwg7sHQJzvWw843gs8db0DVRwf2yS85ECRtTyf5ExLy5lZH+2qbgqeS3TW8t7OajWG4VHZohG8zOjedY8fmMXVMLiUjsygekU3JyCxKRmYxKiddTykyaB0uOBI5ZOk8oNTdN4eFeBhYDMT+kl8M/Eu4/CjwCwv+T1sMPOzujcAHZlYKzHP3lcAOAHevNrN3gOIO55SBzCzoA5I/PmhEb1O3NwySNw6EysangfAPm+zCAyEyfnbwM3JyXOcZARiZE3Q4nD/lwFOPu1Ne3ch7u2qCIAkDZdOuavbUNLGnpoJX3q845FxZaRGKwxApHpFFycgDoVI8MouiXFV/yeCUyOAoBrbGfC8D5ne1j7tHzawKKAzXv9rh2OLYA81sEnASsKqzi5vZUmApwFFHHdXHW5AjJnsUTPl48NOmsQZ2bQiDJHwyWXl78PYWQHoejDvx4CeToukQSYtr0cyMMfmZjMnP5Mxpo9vXt7Y626vq2bizmg/21LKtsp6yfcHPtn117G+IUrq7htLdNZ2eNyM1heJOQqXtyWVMXoYa6mVASmRwdPZffMd6sa72OeyxZpYLPAZ8w933d3Zxd78buBuCqqqeFFgGmIxcOGp+8NMm2gTl7xyo4tqxHl5/AJrDwRMjGTB2RhAohVNh5KTwZ3Lch0xJSbHwF37nnQ6r6pvZtq8+DJS6MFDqKasMlivrmtlcXsvm8tpOj0+PpDBhRGbw1BJWgQVPMNmML8ikKC+DzDT1UZEjL5HBUQZMjPleAmzvYp8yM0sFCoC9hzvWzNIIQuMhd/99YoouA1Zq+oGqqjatLVBRGgTJzvVBmLz7VDBUSqysUcEQ8m1BMnLSge95E+Je7VWQlUZBVhozJnQeWDWN0SBI9tXFPK3UhevqqahtYktFXTii8KFVYQB5makU5WUwJi+DorzM8DMj5jMImJHZaaoWk7hJZON4KkHj+DnANoLG8c+4+4aYfa4HToxpHL/M3a80sxOA/+JA4/hzwDSCN6nuB/a6+zd6WhY1jg9TDVXBHOz7tgQjAe/bEgyZsm8LVG4FbzmwbyQdRhx9cJi0hcvISZB+5IcyqWuKsr2ynq1tTyr7Djy5lFc3Ul7d2D7bYnfSIsbo3AOB0mnI5GcyOjddPe0FSFLjeNhmcQPwDMHruPe6+wYzWwascfflwD3Ag2Hj915gSXjsBjN7hKDROwpc7+4tZnYm8DngTTNbF17qf7r704m6DxnEMgsOfTpp0xKFqq0Hh0lbuGxdFXRgjJU79uAgiQ2X3DEJeeMrOz2VqWPymDqm874u7k5VfTO7wxDZXd3A7v1tywfWlVc3sr8hyo6qBnZ0GH24MyOy0yjKzWBMfgZFuUGojMxJZ2R2OiOz0xiRnc6onHRGZKcxIiud9FT1wB9u1AFQpCN3qN8XBMpBTyofBt/3b+Og5rq0bCiYCHnjIG98F5/jIDV5fT4amlsOCpTyMFAOBEy4vqaRls662h9GbkYqI7LTGJmd3v7ZFiyx69qXc9LJSY+o6myAS9bruCKDk1nwllf2KCg+5dDt0cagqiv2SaXqI6jeBR++AtU7Drz5FStrZOfBkjv2wPfcsXGfPAsgMy3CxFHZ3Y4e3Nrq7KtrOiRQKuua2FfXxL66ZvbVBsuVdc1U1jdT0xilpjHaZW/8zqRHUg4NlpzgaWZkdhp5mWnkZaaSH37mZaaRnxV8z0hNUegkmYJDpLdSM2D01OCnM+5Bv5TqHUFnx5qdB5arw+XyjcFybDtLm+zRMcEytvOwyR6dkIBJSTEKczMozM3g+B5MudLa6lQ3RsNgORAq++qaDwqbyrom9tYeWNfQ3MruMJh6Ky1iBwVKZwETuz6/k/VpGuCyXxQcIvFmFgybklMYjMfVldZWqNvTIVRiQ2YH7HwTancH0/p2lJ4H2SODzpFZo4LP7PAza+SB77Hb4jxacUqKtb89dnQvRoppaG4JQiUMk71tAVPbRGV9M9UNzVQ3RKluiLK/fbmZ/fVRmlpaqahtoqK2qfsLdSErLRIGTSq5mWnkZkTISU8lNyOVnPAnNyMSs9xhXcy+w7GNR8EhkiwpKUHDeu6Yzhvw27REg/G9qndAza5gOPu6iuCppn7vgeWK0qBtpmPDfqy07DBIRsWEzKgOIdNhW3pO3Bv/M9MijC/IYnxB74OsobnlQJCEn9UNUfbXNx+0fn/H9Y1B8FQ3NFPf3EJ9c0ufnng6So+kkBMGyiHBk35o8GSlp5KVFiE7PUJWevCZnR4hMy1Cdnoq2emRAV8dp+AQGegiqQeGaemJaFMQILGhUlcRft97cOBUbg0+GyoPc/10yMgPOlBm5AcjGmcWxCznd1guiFnOC7al58atn0xmWvBLtiivby8buDt1TS0xTzJRahuDn5rws7appX25fV1jS8xyuL6phaaWVprqWg+ZVKw/Uix4KspKTyUrPYXstNROQiYImqz0CNlpQQi17ZOVltq+74wJ+WSnx/dXvYJDZKhJTQ/bRsb2/JiWaBAeHZ9i2gKnYT80VgdPMw37gxcC2pYb93PooBAd2YEQiQ2U2OXY7+m5wcgB6bmHLvezbcfM2p8Kxhf061S4O43R1g4h03JQ4LSvawqWG5paqGtqoa65hfqmKHVNwdNPfbi+Pgyj2qYWaps6aQPrpRX/cDbHjo3v9AUKDhEJnmpyRgc/veUOTTWHhkvj/pjlTtbXlsPezQfWt/Sw2iiS3nWwZOQFVWvt68LvnW7P63cQmVn7E9DoOA6xH21pPShMgnCJtgdLfXPM+qZo+/fY/RuaW6hripKfGd+x20DBISL9ZW1PE/38qzbaeCBgGmugqTYIpMbqmOUaaAq/N9YE69pCa//2g9e1Rnt23ZRUSMsJRgdIyw4/Y7/ndPgMt6dlddi3k2PSsvrUPpQaSSEvkkJeAn7px4OCQ0QGhtSM4KcvTz0duQdB1BYisYHSWNMhhOqCQTKbasPPOmiuDcKoemeH9XV0Xy0Xy2LCJgyU1MwgUNo+Y5cP+syGtExIzerks5PjUjPjPt5aVxQcIjL0mAW/ZNMy4xNEbdyhub7zoDnos+P2mHXRhuAc9fuCN+Wa6w+sizYEP33VFiBtYZI3Hr74x/jdf9tl4n5GEZGhyix4ekjPjm8gxWptDdp7musPDZVDPuuguQGi9Z1/xrnfThsFh4jIQJKSAilZCfulHw/Dr8ujiIj0i4JDRER6RcEhIiK9ouAQEZFeUXCIiEivKDhERKRXFBwiItIrCg4REekVc+/dxPSDkZmVAx8muxw9MBrYk+xCJNBQvj/d2+A1lO+vP/d2tLsXdbZhWATHYGFma9x9brLLkShD+f50b4PXUL6/RN2bqqpERKRXFBwiItIrCo6B5e5kFyDBhvL96d4Gr6F8fwm5N7VxiIhIr+iJQ0REekXBISIivaLgGADMbKKZvWBm75jZBjO7Mdllijczi5jZX83sD8kuS7yZ2Qgze9TM3g3/HS5Idpnixcz+Ifxv8i0z+42ZZSa7TP1hZvea2W4zeytm3Sgz+7OZbQo/RyazjH3Vxb39n/C/yzfM7HEzGxGPayk4BoYo8I/ufjxwGnC9mc1Icpni7UbgnWQXIkF+BvzJ3Y8DZjNE7tPMioGvA3PdfSYQAZYkt1T99itgYYd1NwPPufs04Lnw+2D0Kw69tz8DM919FvAecEs8LqTgGADcfYe7vx4uVxP84ilObqnix8xKgAuBXya7LPFmZvnA2cA9AO7e5O6VyS1VXKUCWWaWCmQD25Ncnn5x95eAvR1WLwbuD5fvBy45ooWKk87uzd1XuHs0/PoqUBKPayk4BhgzmwScBKxKbkni6jbgn4DWZBckAaYA5cB9YVXcL80sJ9mFigd33wb8GPgI2AFUufuK5JYqIca6+w4I/ogDxiS5PInyReCP8TiRgmMAMbNc4DHgG+6+P9nliQczuwjY7e5rk12WBEkFTgbudPeTgFoGb1XHQcK6/sXAZGACkGNmVye3VNIXZvZtgirxh+JxPgXHAGFmaQSh8ZC7/z7Z5YmjM4CLzWwL8DDwSTP7dXKLFFdlQJm7tz0hPkoQJEPB3wAfuHu5uzcDvwdOT3KZEmGXmY0HCD93J7k8cWVm1wAXAZ/1OHXcU3AMAGZmBHXk77j7T5Jdnnhy91vcvcTdJxE0rD7v7kPmr1Z33wlsNbPp4apzgLeTWKR4+gg4zcyyw/9Gz2GINPx3sBy4Jly+BngyiWWJKzNbCHwLuNjd6+J1XgXHwHAG8DmCv8bXhT8XJLtQ0mN/DzxkZm8Ac4AfJrk8cRE+RT0KvA68SfD7YlAPz2FmvwFWAtPNrMzMrgNuBc41s03AueH3QaeLe/sFkAf8Ofy9cldcrqUhR0REpDf0xCEiIr2i4BARkV5RcIiISK8oOEREpFcUHCIi0isKDpEBzMw+PhRHFJbBTcEhIiK9ouAQiQMzu9rMVoedrP4jnH+kxsz+zcxeN7PnzKwo3HeOmb0aM0fCyHD9VDN71szWh8ccE54+N2a+j4fCXtwiSaPgEOknMzseuAo4w93nAC3AZ4Ec4HV3Pxn4b+Cfw0MeAL4VzpHwZsz6h4Db3X02wZhQO8L1JwHfAGYQjMZ7RsJvSuQwUpNdAJEh4BzgFOC18GEgi2CgvFbgt+E+vwZ+b2YFwAh3/+9w/f3A78wsDyh298cB3L0BIDzfancvC7+vAyYBLyf+tkQ6p+AQ6T8D7nf3g2ZXM7PvdtjvcOP7HK76qTFmuQX9fytJpqoqkf57DrjczMZA+xzWRxP8/3V5uM9ngJfdvQrYZ2Znhes/B/x3OP9KmZldEp4jw8yyj+hdiPSQ/nIR6Sd3f9vMvgOsMLMUoBm4nmBSpxPMbC1QRdAOAsHQ3XeFwbAZ+EK4/nPAf5jZsvAcVxzB2xDpMY2OK5IgZlbj7rnJLodIvKmqSkREekVPHCIi0it64hARkV5RcIiISK8oOEREpFcUHCIi0isKDhER6ZX/Dy0gl03GMrOMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens = 784, 10, 256\n",
    "\n",
    "\n",
    "class FlattenLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(FlattenLayer, self).__init__()\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x.view(x.shape[0], -1)\n",
    "\n",
    "\n",
    "net = nn.Sequential(\n",
    "     FlattenLayer(),\n",
    "     nn.Linear(num_inputs, num_hiddens),\n",
    "     nn.ReLU(),\n",
    "     nn.Linear(num_hiddens, num_outputs),\n",
    "     )\n",
    "\n",
    "for params in net.parameters():\n",
    "    init.normal_(params, mean=0, std=0.1)\n",
    "\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr=0.01, weight_decay=0.01)\n",
    "\n",
    "\n",
    "def evaluate_accuracy(test_iter, net):\n",
    "    acc_sum, n = 0, 0\n",
    "    for X, y in test_iter:\n",
    "        y_hat = net(X)\n",
    "        acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "\n",
    "num_epochs = 12\n",
    "\n",
    "\n",
    "def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer):\n",
    "    train_ls, test_ls, x_epoch = [], [], []\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum, train_acc_sum, n = 0.0, 0.0, 0\n",
    "        train_1_test_sum, n_test = 0.0, 0\n",
    "        for X, y in train_iter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y).sum()\n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in test_iter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test).sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "\n",
    "    plt.plot(x_epoch, train_ls, label=\"train_loss\", linewidth=2)\n",
    "    plt.plot(x_epoch, test_ls, label=\"test_loss\", linewidth=1.5)\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.ylabel(\"loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}